#coding:utf-8
"""
*   Coder : Dzlua
*   Email : 505544956@qq.com
*   Time  : 2017/08/16
"""

import threadpool
import urllib2
import codecs
import re

from urlparse import urlparse

#-----------------------
class Spider:
    def __init__(self, threads):
        self.__user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3012.0 Safari/537.36'
        self.__headers = { 'User-Agent' : self.__user_agent }
        self.__pool = threadpool.ThreadPool(threads)
    def __getPage(self, url):
        try:
            request = urllib2.Request(url, headers = self.__headers)
            response = urllib2.urlopen(request)
            pageCode = response.read()#.decode('utf-8')
            return pageCode
        except urllib2.URLError, e:
            if hasattr(e,"reason"):
                err = u"[Spider] 连接 %s 失败,错误原因: %s" % (url, e.reason)
                self.write(err + '\r\n')
                print(err)
                return None
    def put(self, callback, url, args = None):
        def fun(url):
            return self.__getPage(url)
        def funcallback(a, b):
            callback(b, a.args[0], args)
        reqs = threadpool.makeRequests(fun, [url], funcallback)  
        [ self.__pool.putRequest(req) for req in reqs ]
        print('[Spider] add url : %s' % (url))
    def wait(self):
        self.__pool.wait()
    def write(self, text):
        f = codecs.open(self.__class__.__name__ + '.log', 'a', 'utf-8')
        f.write(text)
        f.close()
    def url_parser(self, url):
        return urlparse(url)
    def url_host(self, url):
        ps = self.url_parser(url)
        return ps.scheme + '://' + ps.netloc
    def url_remove_param(self, url):
        return re.sub(r'(\?[a-zA-Z0-9&=]+)?','', url)
    def url_dir(self, url):
        return url[0 : url.rfind('/')]
    def test(self):
        url = 'https://www.oschina.net/project/list/index.html?company=0&sort=score&lang=0&recommend=false&p=2215'
        print(url)
        print(self.url_parser(url))
        print(self.url_host(url))
        print(self.url_remove_param(url))
        print(self.url_dir(url))
#-----------------------
